#coding=utf-8
'''
从大商所、上期所、郑商所、中金所抓取日行情数据并写入粘贴板中
python get_data.py 20181122
'''
from bs4 import BeautifulSoup
import requests
import json
import win32con
import win32clipboard as wincld
import sys

def str2float(s):
    return float(s.strip().replace(',',''))

def str2int(s):
    return int(s.strip().replace(',',''))

#获取期货数据
def get_dce_ftr_data(date):
    print('开始抓取大商所期货数据...')
    postdata = {
            'dayQuotes.variety': 'all',
            'dayQuotes.trade_type': '0'
        }
    postdata['year'] = date[:4]
    postdata['month'] = int(date[4:6]) - 1
    postdata['day'] = date[6:]
    #构造请求URL
    url = 'http://www.dce.com.cn/publicweb/quotesdata/dayQuotesCh.html'
    rsp = requests.post(url, data=postdata)
    content = rsp.content.decode()
    soup = BeautifulSoup(content, features="lxml")
    '''self.date = soup.find(
            'div',
            class_='tradeResult02').find_all('p')[1].text.split('：')[1][0:8]
            '''
    tbl = soup.find('div', class_='dataArea').find('table')
    tr=tbl.find_all('tr')[-1]
    if tr.find('td').text != '总计':
        return None
    
    #成交量
    done_vol = int(str2float(tr.find_all('td')[10].text))#取整
    #持仓量
    position = int(str2float(tr.find_all('td')[11].text))#取整
    #成交额
    turnover = round(str2float(tr.find_all('td')[13].text),2)#保留两位小数
    
    return(done_vol,position,turnover)

#获取期权数据
def get_dce_opt_data(date=None):
    print('开始抓取大商所期权数据...')
    postdata = {
        'dayQuotes.variety': 'all',
        'dayQuotes.trade_type': '1'
    }
    postdata['year'] = date[:4]
    postdata['month'] = int(date[4:6]) - 1
    postdata['day'] = date[6:]
    #构造请求URL
    url = 'http://www.dce.com.cn/publicweb/quotesdata/dayQuotesCh.html'
    rsp = requests.post(url, data=postdata)
    content = rsp.content.decode()
    soup = BeautifulSoup(content, features="lxml")

    '''date = soup.find(
        'div',
        class_='tradeResult02').find_all('p')[1].text.split('：')[1][0:8]'''
    
    tbl = soup.find('div', class_='dataArea').find('table')
    tr=tbl.find_all('tr')[-1]
    if tr.find('td').text != '总计':
        return None
    
    #成交量
    done_vol = int(str2float(tr.find_all('td')[11].text))#取整
    #持仓量
    position = int(str2float(tr.find_all('td')[12].text))#取整
    #成交额
    turnover = round(str2float(tr.find_all('td')[14].text),2)#四舍五入保留两位小数
    
    return(done_vol,position,turnover)


#获取期货数据
def get_shfe_ftr_data(date):
    print('开始抓取上期所期货数据...')
    url='http://www.shfe.com.cn/data/dailydata/kx/kx{}.dat?isAjax=true'.format(date)
    rsp = requests.get(url)
    content = rsp.content.decode()
    ret=json.loads(content)
    #持仓量，总计1不包含自对冲，为列表中的倒数第二个
    position=int(ret['o_curinstrument'][-2]['OPENINTEREST'])#取整
    
    #总计1不包含自对冲，为列表中的倒数第二个
    sum_data = ret['o_curproduct'][-2]
    
    #成交量
    done_vol = round(sum_data['VOLUME'],2)#四舍五入保留两位小数
    #成交额    
    turnover = round(sum_data['TURNOVER'],2)#四舍五入保留两位小数
    
    #年成交量
    year_done_vol = round(sum_data['YEARVOLUME'],2)#四舍五入保留两位小数
    
    #年成交额
    year_turnover = round(sum_data['YEARTURNOVER'],2)#四舍五入保留两位小数
    
    #print(done_vol,position,turnover,year_done_vol,year_turnover)
    
    return(done_vol,position,turnover,year_done_vol,year_turnover)
  
  
#获取期权数据
def get_shfe_opt_data(date):
    print('开始抓取上期所期权数据...')
    url='http://www.shfe.com.cn/data/dailydata/option/kx/kx{}.dat?isAjax=true'.format(date)
    rsp = requests.get(url)
    content = rsp.content.decode()
    ret=json.loads(content)
    
    #持仓量，总计是最后一个
    position=int(ret['o_curinstrument'][-1]['OPENINTEREST'])#取整
    
    #总计是最后一个
    sum_data = ret['o_curproduct'][-1]
    
    #成交量
    done_vol = round(sum_data['VOLUME'],2)#四舍五入保留两位小数
    #成交额    
    turnover = round(sum_data['TURNOVER'],2)#四舍五入保留两位小数
    
    #年成交量
    year_done_vol = round(sum_data['YEARVOLUME'],2)#四舍五入保留两位小数
    
    #年成交额
    year_turnover = round(sum_data['YEARTURNOVER'],2)#四舍五入保留两位小数
    
    #print(done_vol,position,turnover,year_done_vol,year_turnover)
    
    return(done_vol,position,turnover,year_done_vol,year_turnover)  
            
            
#获取上海能源中心数据
def get_ine_data(date):
    print('开始抓取上海能源中心数据...')
    url='http://www.ine.cn/data/dailydata/kx/kx{}.dat'.format(date)
    rsp = requests.get(url)
    content = rsp.content.decode()
    ret=json.loads(content)
    
    #持仓量，总计是最后一个
    position=int(ret['o_curinstrument'][-1]['OPENINTEREST'])#取整
    
    #总计是最后一个
    sum_data = ret['o_curproduct'][-1]
    
    #成交量
    done_vol = round(sum_data['VOLUME'],2)#四舍五入保留两位小数
    #成交额    
    turnover = round(sum_data['TURNOVER'],6)#四舍五入保留六位小数
    
    #年成交量
    year_done_vol = round(sum_data['YEARVOLUME'],4)#四舍五入保留四位小数
    
    #年成交额
    year_turnover = round(sum_data['YEARTURNOVER'],5)#四舍五入保留五位小数    
    
    #print(done_vol,position,turnover,year_done_vol,year_turnover)
    
    return(done_vol,position,turnover,year_done_vol,year_turnover)  
    
    
#获取郑商所期货数据
def get_czce_ftr_data(date):
    print('开始抓取郑商所期货数据...')
    url='http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataDaily.htm'.format(date[0:4],date)
    rsp = requests.get(url)
    content = rsp.content.decode()
    soup = BeautifulSoup(content, features="lxml")
    #总计在倒数第二个
    tr = soup.find('table', id='tab1').find_all('tr')[-2]

    #成交量
    done_vol = str2int(tr.find_all('td')[9].text)#取整    
    #持仓量
    position=str2int(tr.find_all('td')[10].text)#取整    
    #成交额    
    turnover = round(str2float(tr.find_all('td')[12].text),2)#四舍五入保留两位小数
    
    year_done_vol = 0
    year_turnover = 0
    with open('FutureDataHistory.txt',encoding='utf-8') as f:
        count=0
        for l in f:
            if count < 2:
                count += 1
                continue            
            ret=l.strip().split('|')
            try:
                data_date=ret[0].replace('-','')
                if data_date > date:
                    break
                year_done_vol += str2int(ret[10])#取整
                year_turnover += round(str2float(ret[13]),2)#四舍五入取两位小数
            except Exception as e:
                pass#表头忽略
    return (done_vol,position,turnover,year_done_vol,year_turnover)
   
#获取郑商所期权数据
def get_czce_opt_data(date):
    print('开始抓取郑商所期权数据...')
    url='http://www.czce.com.cn/cn/DFSStaticFiles/Option/{}/{}/OptionDataDaily.htm'.format(date[0:4],date)
    rsp = requests.get(url)
    content = rsp.content.decode()
    soup = BeautifulSoup(content, features="lxml")
    #总计在倒数第二个
    tr = soup.find('table', id='tab1').find_all('tr')[-2]

    #成交量
    done_vol = str2int(tr.find_all('td')[9].text)#取整    
    #持仓量
    position=str2int(tr.find_all('td')[10].text)#取整    
    #成交额    
    turnover = round(str2float(tr.find_all('td')[12].text),2)#四舍五入保留两位小数    
    
    year_done_vol = 0
    year_turnover = 0
    with open('OptionDataHistory.txt',encoding='utf-8') as f:
        count=0
        for l in f:
            if count < 2:
                count += 1
                continue
            ret=l.strip().split('|')
            try:
                data_date=ret[0].replace('-','')
                if data_date > date:
                    break                
                year_done_vol += str2int(ret[10])#取整
                year_turnover += round(str2float(ret[13]),2)#四舍五入取两位小数
            except Exception as e:
                pass#表头忽略
            
    return (done_vol,position,turnover,year_done_vol,year_turnover)    
    
    
    
#获取中金所数据

def get_cffex_data(date):
    print('开始抓取中金所数据...')
    url='http://www.cffex.com.cn/sj/hqsj/rtj/{}/{}/index.xml'.format(date[0:6],date[6:])
    rsp = requests.get(url)
    content = rsp.content.decode()
    soup = BeautifulSoup(content, features="lxml")   
    done_vol=0
    position=0
    turnover=0
    for data in soup.find_all('dailydata'):
        done_vol += str2int(data.volume.string)
        turnover += str2int(data.turnover.string)
        position += str2int(data.openinterest.string)
        
    #print(done_vol,position,turnover)
    return (done_vol,position,turnover)
    
    
    
if __name__ == '__main__':
    if len(sys.argv) != 2:
        print(__doc__)
        sys.exit(1)
    
    date=sys.argv[1]
    
    print('开始抓取数据...交易日期{}\n'.format(date))
    
    #大商所
    done_vol_ftr,position_ftr,turnover_ftr = get_dce_ftr_data(date)
    done_vol_opt,position_opt,turnover_opt = get_dce_opt_data(date)
    
    done_vol = done_vol_ftr+done_vol_opt
    position = position_ftr+position_opt
    turnover = turnover_ftr+turnover_opt
    info = '{}\t{}\t{}\n'.format(done_vol,position,turnover)
    
    #上期所
    done_vol_ftr,position_ftr,turnover_ftr,year_done_vol_ftr,year_turnover_ftr = get_shfe_ftr_data(date)
    done_vol_opt,position_opt,turnover_opt,year_done_vol_opt,year_turnover_opt = get_shfe_opt_data(date)
    done_vol = done_vol_ftr+done_vol_opt
    position = position_ftr+position_opt
    turnover = turnover_ftr+turnover_opt    
    year_done_vol = year_done_vol_ftr+year_done_vol_opt
    year_turnover = year_turnover_ftr+year_turnover_opt
    
	#能源交易中心不需要抓取
    '''
	done_vol_ftr,position_ftr,turnover_ftr,year_done_vol_ftr,year_turnover_ftr = get_ine_data(date)    
    done_vol = done_vol_ftr+done_vol
    position = position_ftr+position
    turnover = turnover_ftr+turnover    
    year_done_vol = year_done_vol_ftr+year_done_vol
    year_turnover = year_turnover_ftr+year_turnover   
	''' 
    info = info+'{}\t{}\t{}\t{}\t{}\n'.format(done_vol,position,turnover,year_done_vol,year_turnover)
    
    #郑商所
    done_vol_ftr,position_ftr,turnover_ftr,year_done_vol_ftr,year_turnover_ftr = get_czce_ftr_data(date)
    done_vol_opt,position_opt,turnover_opt,year_done_vol_opt,year_turnover_opt = get_czce_opt_data(date)
    done_vol = done_vol_ftr+done_vol_opt
    position = position_ftr+position_opt
    turnover = turnover_ftr+turnover_opt     
    year_done_vol = year_done_vol_ftr+year_done_vol_opt
    year_turnover = year_turnover_ftr+year_turnover_opt    
    info = info+'{}\t{}\t{}\t{}\t{}\n'.format(done_vol,position,turnover,year_done_vol,year_turnover)
    
    #中金所
    done_vol,position,turnover = get_cffex_data(date)
    info = info+'{}\t{}\t{}'.format(done_vol,position,turnover)
    
    print('数据抓取完毕，开始写入粘贴板...')
    wincld.OpenClipboard()
    wincld.EmptyClipboard()
    wincld.SetClipboardData(win32con.CF_UNICODETEXT, info)
    wincld.CloseClipboard()    
    print('='*8)
    print('写入完成，请打开excel并粘贴')